﻿using System;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using LSA.Framework.Pinvoke;
using System.IO;
using System.Text.RegularExpressions;
using System.Linq;

namespace LSA.Framework
{
    /// <summary>
    /// Builds a <see cref="SparseMatrix"/> from a collection of text files.
    /// </summary>
    [Serializable]
    public unsafe class FrequencyMatrixBuilder
    {
        private int values;

        private Dictionary<string, int> wordsList;
        public Dictionary<int, bool> EntitiesCache;

        public Dictionary<string, int> WordsList // TODO: change to ulong
        {
            get { return wordsList; }
            private set { wordsList = value; }
        }

        private Dictionary<int, int>[] wordsFrequency;
        public Dictionary<int, int>[] WordsFrequency
        {
            get { return wordsFrequency; }
            private set { wordsFrequency = value; }
        }

        public int DocumentsCount;
        public int[] WordsPerDocument;
        public bool TfIdf;
        public bool Stem;

        [NonSerialized]
        private SparseMatrix* matrix;

        public SparseMatrix* Matrix
        {
            get { return matrix; }
            private set { matrix = value; }
        }

        public FrequencyMatrixBuilder(string[] fileNames, bool tfIdf, bool stem, Dictionary<string, bool> stopWords, string[] entities)
        {
            DocumentsCount = fileNames.Length;
            WordsList = new Dictionary<string, int>(); // TODO: change to ulong
            WordsFrequency = new Dictionary<int, int>[DocumentsCount]; // TODO:
            WordsPerDocument = new int[DocumentsCount];
            TfIdf = tfIdf;
            Stem = stem;

            values = 0; // TODO:

            AnalyzeDocuments(fileNames, stopWords, entities);

            Matrix = CppInvoker.BuildNewSparseMatrix(WordsList.Count, DocumentsCount, values);
            if (Matrix == null)
            {
                throw new Exception("Unknow error occured. Could not built new sparse matrix.");
            }

            FillSparseMatrix();
        }

        private void AnalyzeDocuments(string[] fileNames, Dictionary<string, bool> stopWords, string[] entities)
        {
            int nextWordIndex = 0; // TODO:
            EntitiesCache = new Dictionary<int, bool>();

            Parallel.For(0, DocumentsCount, delegate(int i)
            {
                string fileName = fileNames[i];

                using (StreamReader stream = File.OpenText(fileName))
                {
                    // Changed to ReadToEnd instead of ReadLine, because of slow entities replacements
                    string lineString = stream.ReadToEnd();
                    string line = lineString.ToLower();

                    // Begin entities replace in text
                    for (int j = 0; j < entities.Length; j++)
                    {
                        if (entities[j].Length < 2)
                        {
                            continue;
                        }
                        line = line.Replace(entities[j].ToLower(), "@!@" + j); // FIX
                    }

                    // Analyze every word in the document 
                    foreach (Match wordMatch in Regex.Matches(line, @"(http://(.*?)/)|(\w(?<!\d)[\w\-]*[\w])|@!@\d+", RegexOptions.Compiled))
                    {
                        bool entity = wordMatch.Value.Contains(@"@!@");

                        string word;
                        if(entity)
                            word = wordMatch.Value.Replace("@!@", "");
                        else
                        {
                            if(wordMatch.Value.Contains("http://"))
                                continue;
                            word = wordMatch.Value;
                        }
                       
                        bool retValue;
                        int wordIndex; // TODO:

                        if (stopWords.TryGetValue(word, out retValue))
                            continue;
                        if (!WordsList.TryGetValue(word, out wordIndex))
                        {
                            if (Stem)
                            {
                                if (!entity)
                                {
                                    word = PorterStemmer.Stem(word);
                                }
                            }

                            lock (this)
                            {
                                if (!WordsList.TryGetValue(word, out wordIndex))
                                {
                                    WordsList[word] = nextWordIndex;
                                    wordIndex = nextWordIndex;
                                        
                                    if (entity)
                                    {
                                        EntitiesCache[wordIndex] = true;
                                    }
                                    Interlocked.Increment(ref nextWordIndex);
                                }
                            }
                        }

                        if (WordsFrequency[i] == null)
                        {
                            WordsFrequency[i] = new Dictionary<int, int>();
                            WordsFrequency[i][wordIndex] = 1;
                            

                                Interlocked.Increment(ref values);
                                Interlocked.Increment(ref WordsPerDocument[i]);
                            
                        }
                        else
                        {
                            int retValue2; // TODO:
                            if (!WordsFrequency[i].TryGetValue(wordIndex, out retValue2))
                            {
                                WordsFrequency[i][wordIndex] = 1;
                                
                                    Interlocked.Increment(ref values);
                                    Interlocked.Increment(ref WordsPerDocument[i]);
                            }
                            else
                            {
                                WordsFrequency[i][wordIndex] += 1;
                                Interlocked.Increment(ref WordsPerDocument[i]);
                            }
                        }
                    }
                }
            });
        }

        private void FillSparseMatrix()
        {
            int v = 0;
            for (int c = 0; c < DocumentsCount; c++)
            {
                Matrix->pointr[c] = v;
                if (WordsFrequency[c] == null) continue;

                foreach (KeyValuePair<int, int> word in WordsFrequency[c])
                {
                    int totalDocumentWithWord = (TfIdf) ? WordsFrequency.Count(val => val.ContainsKey(word.Key)) : 0;
                    Matrix->rowind[v] = word.Key;
                    Matrix->value[v] = (TfIdf)
                                           ? (word.Value / (double)WordsPerDocument[c]) *
                                             Math.Log10(DocumentsCount / (double)totalDocumentWithWord)
                                           : word.Value;
                    v++;
                }
            }
            Matrix->pointr[DocumentsCount] = values;
        }

        [OnDeserialized]
        internal void OnDeserializedMethod(StreamingContext context)
        {
            wordsList.OnDeserialization(this);
            foreach (Dictionary<int, int> dictionary in wordsFrequency)
            {
                dictionary.OnDeserialization(this);
            }

            Matrix = CppInvoker.BuildNewSparseMatrix(WordsList.Count, DocumentsCount, values);
            if (Matrix == null)
            {
                throw new Exception("Unknow error occured. Could not built new sparse matrix.");
            }

            FillSparseMatrix();
        }
    }
}
